{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "初始化Q表完成\n",
      "Q_Table,0\n",
      "episode is 0\n",
      "Random_start:18\n",
      "419\n",
      "0.995860578145\n",
      "episode is 1\n",
      "Normal_start:1\n",
      "736\n",
      "0.992740348289\n",
      "episode is 2\n",
      "Normal_start:1\n",
      "1027\n",
      "0.989884730814\n",
      "episode is 3\n",
      "Random_start:48\n",
      "1542\n",
      "0.984851296669\n",
      "episode is 4\n",
      "Random_start:75\n",
      "2083\n",
      "0.979591591485\n",
      "episode is 5\n",
      "Random_start:30\n",
      "2480\n",
      "0.975749943583\n",
      "episode is 6\n",
      "Random_start:96\n",
      "3037\n",
      "0.970385669769\n",
      "episode is 7\n",
      "Random_start:29\n",
      "3247\n",
      "0.968370976031\n",
      "episode is 8\n",
      "Normal_start:1\n",
      "3849\n",
      "0.962618913834\n",
      "episode is 9\n",
      "Normal_start:1\n",
      "4466\n",
      "0.956759350477\n",
      "episode is 10\n",
      "Random_start:72\n",
      "4615\n",
      "0.955349729474\n",
      "episode is 11\n",
      "Random_start:42\n",
      "4853\n",
      "0.953102472414\n",
      "episode is 12\n",
      "Random_start:14\n",
      "5613\n",
      "0.945962061554\n",
      "episode is 13\n",
      "Normal_start:1\n",
      "6023\n",
      "0.942132473123\n",
      "episode is 14\n",
      "Normal_start:1\n",
      "6548\n",
      "0.937251601139\n",
      "episode is 15\n",
      "Normal_start:1\n",
      "7710\n",
      "0.926539296558\n",
      "episode is 16\n",
      "Normal_start:1\n",
      "7828\n",
      "0.925458418032\n",
      "episode is 17\n",
      "Random_start:92\n",
      "7943\n",
      "0.924406245966\n",
      "episode is 18\n",
      "Normal_start:1\n",
      "8742\n",
      "0.917129250372\n",
      "episode is 19\n",
      "Random_start:65\n",
      "9860\n",
      "0.9070440268\n",
      "episode is 20\n",
      "Normal_start:1\n",
      "10143\n",
      "0.904508980986\n",
      "episode is 21\n",
      "Normal_start:1\n",
      "10349\n",
      "0.902668189152\n",
      "episode is 22\n",
      "Normal_start:1\n",
      "10455\n",
      "0.901722462195\n",
      "episode is 23\n",
      "Normal_start:1\n",
      "10617\n",
      "0.900279041293\n",
      "episode is 24\n",
      "Random_start:56\n",
      "10678\n",
      "0.89973613668\n",
      "episode is 25\n",
      "Normal_start:1\n",
      "11247\n",
      "0.894687913877\n",
      "episode is 26\n",
      "Random_start:86\n",
      "11348\n",
      "0.893794830167\n",
      "episode is 27\n",
      "Random_start:46\n",
      "11388\n",
      "0.893441382929\n",
      "episode is 28\n",
      "Random_start:11\n",
      "11613\n",
      "0.891455874352\n",
      "episode is 29\n",
      "Random_start:43\n",
      "11679\n",
      "0.890874305414\n",
      "episode is 30\n",
      "Normal_start:1\n",
      "11854\n",
      "0.889334123432\n",
      "episode is 31\n",
      "Random_start:82\n",
      "12037\n",
      "0.887726413489\n",
      "episode is 32\n",
      "Random_start:90\n",
      "12650\n",
      "0.882362408048\n",
      "episode is 33\n",
      "Random_start:18\n",
      "12872\n",
      "0.880427911588\n",
      "episode is 34\n",
      "Normal_start:1\n",
      "13164\n",
      "0.877889969285\n",
      "episode is 35\n",
      "Random_start:28\n",
      "13266\n",
      "0.877005172839\n",
      "episode is 36\n",
      "Random_start:8\n",
      "13804\n",
      "0.87235321001\n",
      "episode is 37\n",
      "Random_start:69\n",
      "13820\n",
      "0.872215244534\n",
      "episode is 38\n",
      "Normal_start:1\n",
      "13949\n",
      "0.871103703966\n",
      "episode is 39\n",
      "Random_start:53\n",
      "14314\n",
      "0.867966404501\n",
      "episode is 40\n",
      "Random_start:12\n",
      "14539\n",
      "0.866038150191\n",
      "episode is 41\n",
      "Random_start:31\n",
      "14719\n",
      "0.864498667471\n",
      "episode is 42\n",
      "Random_start:8\n",
      "14839\n",
      "0.863473884063\n",
      "episode is 43\n",
      "Normal_start:1\n",
      "15595\n",
      "0.857045949706\n",
      "episode is 44\n",
      "Normal_start:1\n",
      "15714\n",
      "0.856038564539\n",
      "episode is 45\n",
      "Random_start:78\n",
      "15770\n",
      "0.855564915577\n",
      "episode is 46\n",
      "Random_start:20\n",
      "16193\n",
      "0.851995730133\n",
      "episode is 47\n",
      "Random_start:32\n",
      "16227\n",
      "0.851709500247\n",
      "episode is 48\n",
      "Normal_start:1\n",
      "16309\n",
      "0.851019581362\n",
      "episode is 49\n",
      "Normal_start:1\n",
      "16410\n",
      "0.850170580402\n",
      "episode is 50\n",
      "Random_start:34\n",
      "16652\n",
      "0.848139825802\n",
      "episode is 51\n",
      "Random_start:16\n",
      "16910\n",
      "0.845980212151\n",
      "episode is 52\n",
      "Normal_start:1\n",
      "17058\n",
      "0.844743876551\n",
      "episode is 53\n",
      "Random_start:34\n",
      "17358\n",
      "0.842243397515\n",
      "episode is 54\n",
      "Normal_start:1\n",
      "17685\n",
      "0.839526406307\n",
      "episode is 55\n",
      "Random_start:38\n",
      "17748\n",
      "0.839003969256\n",
      "episode is 56\n",
      "Random_start:80\n",
      "17816\n",
      "0.838440438179\n",
      "episode is 57\n",
      "Normal_start:1\n",
      "17893\n",
      "0.83780278457\n",
      "episode is 58\n",
      "Normal_start:1\n",
      "18060\n",
      "0.836421507607\n",
      "episode is 59\n",
      "Random_start:40\n",
      "18111\n",
      "0.836000140096\n",
      "episode is 60\n",
      "Random_start:41\n",
      "18149\n",
      "0.835686319672\n",
      "episode is 61\n",
      "Random_start:46\n",
      "18478\n",
      "0.83297427544\n",
      "episode is 62\n",
      "Normal_start:1\n",
      "18697\n",
      "0.83117393387\n",
      "episode is 63\n",
      "Random_start:72\n",
      "18861\n",
      "0.82982831233\n",
      "episode is 64\n",
      "Random_start:14\n",
      "19011\n",
      "0.828599491707\n",
      "episode is 65\n",
      "Normal_start:1\n",
      "19145\n",
      "0.827503302999\n",
      "episode is 66\n",
      "Normal_start:1\n",
      "19233\n",
      "0.826784216537\n",
      "episode is 67\n",
      "Normal_start:1\n",
      "19358\n",
      "0.825763874113\n",
      "episode is 68\n",
      "Random_start:80\n",
      "19389\n",
      "0.825511026505\n",
      "episode is 69\n",
      "Normal_start:1\n",
      "19483\n",
      "0.82474480632\n",
      "episode is 70\n",
      "Random_start:57\n",
      "19514\n",
      "0.824492274575\n",
      "episode is 71\n",
      "Normal_start:1\n",
      "19628\n",
      "0.823564282438\n",
      "episode is 72\n",
      "Random_start:51\n",
      "19738\n",
      "0.822669853753\n",
      "episode is 73\n",
      "Normal_start:1\n",
      "19971\n",
      "0.820778537233\n",
      "episode is 74\n",
      "Random_start:46\n",
      "20063\n",
      "0.820032963995\n",
      "episode is 75\n",
      "Normal_start:1\n",
      "20149\n",
      "0.81933663511\n",
      "episode is 76\n",
      "Random_start:24\n",
      "20248\n",
      "0.818535788326\n",
      "episode is 77\n",
      "Random_start:42\n",
      "20303\n",
      "0.818091215911\n",
      "episode is 78\n",
      "Random_start:24\n",
      "20395\n",
      "0.817348113872\n",
      "episode is 79\n",
      "Random_start:24\n",
      "20641\n",
      "0.815364478384\n",
      "episode is 80\n",
      "Random_start:20\n",
      "20659\n",
      "0.815219525824\n",
      "episode is 81\n",
      "Random_start:81\n",
      "20764\n",
      "0.814374489044\n",
      "episode is 82\n",
      "Random_start:23\n",
      "20832\n",
      "0.81382770032\n",
      "episode is 83\n",
      "Random_start:10\n",
      "20941\n",
      "0.812952005467\n",
      "episode is 84\n",
      "Random_start:86\n",
      "20949\n",
      "0.812887771876\n",
      "episode is 85\n",
      "Normal_start:1\n",
      "21144\n",
      "0.81132366622\n",
      "episode is 86\n",
      "Normal_start:1\n",
      "21198\n",
      "0.810891068252\n",
      "episode is 87\n",
      "Normal_start:1\n",
      "21266\n",
      "0.81034664745\n",
      "episode is 88\n",
      "Random_start:21\n",
      "21319\n",
      "0.809922576115\n",
      "episode is 89\n",
      "Normal_start:1\n",
      "21588\n",
      "0.807773675952\n",
      "episode is 90\n",
      "Normal_start:1\n",
      "21779\n",
      "0.806251382484\n",
      "episode is 91\n",
      "Random_start:17\n",
      "21859\n",
      "0.805614636111\n",
      "episode is 92\n",
      "Random_start:18\n",
      "21934\n",
      "0.805018148844\n",
      "episode is 93\n",
      "Normal_start:1\n",
      "22019\n",
      "0.804342670537\n",
      "episode is 94\n",
      "Normal_start:1\n",
      "22113\n",
      "0.803596339257\n",
      "episode is 95\n",
      "Random_start:89\n",
      "22115\n",
      "0.803580467489\n",
      "episode is 96\n",
      "Random_start:73\n",
      "22160\n",
      "0.803223436617\n",
      "episode is 97\n",
      "Normal_start:1\n",
      "22197\n",
      "0.802929998234\n",
      "episode is 98\n",
      "Random_start:57\n",
      "22568\n",
      "0.799993678183\n",
      "episode is 99\n",
      "Random_start:60\n",
      "22580\n",
      "0.799898884629\n",
      "episode is 100\n",
      "Normal_start:1\n",
      "22734\n",
      "0.798683376528\n",
      "episode is 101\n",
      "Normal_start:1\n",
      "22892\n",
      "0.79743824071\n",
      "episode is 102\n",
      "Random_start:59\n",
      "22908\n",
      "0.79731226067\n",
      "episode is 103\n",
      "Normal_start:1\n",
      "23115\n",
      "0.795684209904\n",
      "episode is 104\n",
      "Normal_start:1\n",
      "23168\n",
      "0.795267907603\n",
      "episode is 105\n",
      "Random_start:89\n",
      "23172\n",
      "0.795236497515\n",
      "episode is 106\n",
      "Normal_start:1\n",
      "23304\n",
      "0.794200669135\n",
      "episode is 107\n",
      "Random_start:37\n",
      "23340\n",
      "0.793918407704\n",
      "episode is 108\n",
      "Normal_start:1\n",
      "23538\n",
      "0.79236778488\n",
      "episode is 109\n",
      "Random_start:41\n",
      "23615\n",
      "0.791765593559\n",
      "episode is 110\n",
      "Normal_start:1\n",
      "23650\n",
      "0.791492023479\n",
      "episode is 111\n",
      "Random_start:88\n",
      "23763\n",
      "0.790609436248\n",
      "episode is 112\n",
      "Normal_start:1\n",
      "23925\n",
      "0.789345872724\n",
      "episode is 113\n",
      "Normal_start:1\n",
      "23982\n",
      "0.788901772157\n",
      "episode is 114\n",
      "Random_start:54\n",
      "24153\n",
      "0.787570988272\n",
      "episode is 115\n",
      "Normal_start:1\n",
      "24257\n",
      "0.786762734808\n",
      "episode is 116\n",
      "Normal_start:1\n",
      "24441\n",
      "0.785334805474\n",
      "episode is 117\n",
      "Random_start:98\n",
      "24454\n",
      "0.785234018501\n",
      "episode is 118\n",
      "Random_start:82\n",
      "24550\n",
      "0.784490150957\n",
      "episode is 119\n",
      "Random_start:16\n",
      "24609\n",
      "0.784033336541\n",
      "episode is 120\n",
      "Normal_start:1\n",
      "24852\n",
      "0.782154718978\n",
      "episode is 121\n",
      "Normal_start:1\n",
      "24897\n",
      "0.781807327523\n",
      "episode is 122\n",
      "Random_start:88\n",
      "24900\n",
      "0.781784173651\n",
      "episode is 123\n",
      "Normal_start:1\n",
      "24954\n",
      "0.781367522703\n",
      "episode is 124\n",
      "Normal_start:1\n",
      "25064\n",
      "0.780519484934\n",
      "episode is 125\n",
      "Random_start:91\n",
      "25090\n",
      "0.780319175909\n",
      "episode is 126\n",
      "Random_start:22\n",
      "25188\n",
      "0.779564632904\n",
      "episode is 127\n",
      "Random_start:46\n",
      "25303\n",
      "0.778680142255\n",
      "episode is 128\n",
      "Normal_start:1\n",
      "25368\n",
      "0.778180662511\n",
      "episode is 129\n",
      "Random_start:23\n",
      "25472\n",
      "0.77738216991\n",
      "episode is 130\n",
      "Normal_start:1\n",
      "25623\n",
      "0.776224297248\n",
      "episode is 131\n",
      "Random_start:55\n",
      "25731\n",
      "0.775397221708\n",
      "episode is 132\n",
      "Random_start:42\n",
      "25777\n",
      "0.775045219953\n",
      "episode is 133\n",
      "Random_start:12\n",
      "25882\n",
      "0.774242344055\n",
      "episode is 134\n",
      "Random_start:28\n",
      "25895\n",
      "0.774142999008\n",
      "episode is 135\n",
      "Random_start:31\n",
      "25940\n",
      "0.773799212016\n",
      "episode is 136\n",
      "Random_start:16\n",
      "26023\n",
      "0.773165521688\n",
      "episode is 137\n",
      "Random_start:31\n",
      "26158\n",
      "0.772135943356\n",
      "episode is 138\n",
      "Normal_start:1\n",
      "26247\n",
      "0.771457944121\n",
      "episode is 139\n",
      "Random_start:95\n",
      "26294\n",
      "0.771100142977\n",
      "episode is 140\n",
      "Random_start:17\n",
      "26343\n",
      "0.770727295262\n",
      "episode is 141\n",
      "Random_start:87\n",
      "26384\n",
      "0.770415461001\n",
      "episode is 142\n",
      "Random_start:80\n",
      "26386\n",
      "0.770400252844\n",
      "episode is 143\n",
      "Random_start:29\n",
      "26424\n",
      "0.770111355642\n",
      "episode is 144\n",
      "Random_start:57\n",
      "26506\n",
      "0.76948831981\n",
      "episode is 145\n",
      "Normal_start:1\n",
      "26598\n",
      "0.768789911872\n",
      "episode is 146\n",
      "Random_start:8\n",
      "26658\n",
      "0.76833477448\n",
      "episode is 147\n",
      "Random_start:94\n",
      "26815\n",
      "0.767145123005\n",
      "episode is 148\n",
      "Random_start:12\n",
      "26912\n",
      "0.76641104832\n",
      "episode is 149\n",
      "Normal_start:1\n",
      "26990\n",
      "0.765821277742\n",
      "episode is 150\n",
      "Random_start:78\n",
      "26994\n",
      "0.765791045496\n",
      "episode is 151\n",
      "Normal_start:1\n",
      "27036\n",
      "0.765473679908\n",
      "episode is 152\n",
      "Normal_start:1\n",
      "27160\n",
      "0.764537473113\n",
      "episode is 153\n",
      "Normal_start:1\n",
      "27281\n",
      "0.763625034907\n",
      "episode is 154\n",
      "Random_start:18\n",
      "27309\n",
      "0.763414049437\n",
      "episode is 155\n",
      "Random_start:58\n",
      "27336\n",
      "0.763210655103\n",
      "episode is 156\n",
      "Random_start:40\n",
      "27350\n",
      "0.763105212992\n",
      "episode is 157\n",
      "Normal_start:1\n",
      "27442\n",
      "0.762412674813\n",
      "episode is 158\n",
      "Normal_start:1\n",
      "27631\n",
      "0.760991957858\n",
      "episode is 159\n",
      "Normal_start:1\n",
      "27693\n",
      "0.760526487155\n",
      "episode is 160\n",
      "Random_start:31\n",
      "27717\n",
      "0.760346382411\n",
      "episode is 161\n",
      "Normal_start:1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27876\n",
      "0.759154279636\n",
      "episode is 162\n",
      "Normal_start:1\n",
      "28049\n",
      "0.757859363158\n",
      "episode is 163\n",
      "Random_start:68\n",
      "28068\n",
      "0.757717283377\n",
      "episode is 164\n",
      "Random_start:40\n",
      "28077\n",
      "0.75764999185\n",
      "episode is 165\n",
      "Random_start:29\n",
      "28144\n",
      "0.757149234128\n",
      "episode is 166\n",
      "Normal_start:1\n",
      "28270\n",
      "0.756208418931\n",
      "episode is 167\n",
      "Random_start:82\n",
      "28342\n",
      "0.75567134224\n",
      "episode is 168\n",
      "Random_start:35\n",
      "28539\n",
      "0.754203815684\n",
      "episode is 169\n",
      "Random_start:67\n",
      "28580\n",
      "0.753898754661\n",
      "episode is 170\n",
      "Normal_start:1\n",
      "28694\n",
      "0.753051193283\n",
      "episode is 171\n",
      "Normal_start:1\n",
      "28731\n",
      "0.752776315197\n",
      "episode is 172\n",
      "Random_start:41\n",
      "28812\n",
      "0.752174909984\n",
      "episode is 173\n",
      "Random_start:11\n",
      "28879\n",
      "0.751677819338\n",
      "episode is 174\n",
      "Normal_start:1\n",
      "29021\n",
      "0.75062538424\n",
      "episode is 175\n",
      "Normal_start:1\n",
      "29120\n",
      "0.749892527933\n",
      "episode is 176\n",
      "Random_start:49\n",
      "29259\n",
      "0.748864791762\n",
      "episode is 177\n",
      "Random_start:66\n",
      "29266\n",
      "0.748813073037\n",
      "episode is 178\n",
      "Random_start:99\n",
      "29273\n",
      "0.748761357931\n",
      "episode is 179\n",
      "Random_start:33\n",
      "29374\n",
      "0.748015585638\n",
      "episode is 180\n",
      "Random_start:90\n",
      "29376\n",
      "0.748000825474\n",
      "episode is 181\n",
      "Random_start:83\n",
      "29477\n",
      "0.747255820931\n",
      "episode is 182\n",
      "Normal_start:1\n",
      "29511\n",
      "0.747005196561\n",
      "episode is 183\n",
      "Random_start:46\n",
      "29545\n",
      "0.746754657388\n",
      "episode is 184\n",
      "Normal_start:1\n",
      "29732\n",
      "0.745378213555\n",
      "episode is 185\n",
      "Random_start:62\n",
      "29818\n",
      "0.744746060156\n",
      "episode is 186\n",
      "Random_start:87\n",
      "29862\n",
      "0.744422843002\n",
      "episode is 187\n",
      "Normal_start:1\n",
      "29961\n",
      "0.743696124173\n",
      "episode is 188\n",
      "Normal_start:1\n",
      "30068\n",
      "0.742911489175\n",
      "episode is 189\n",
      "Normal_start:1\n",
      "30129\n",
      "0.742464549497\n",
      "episode is 190\n",
      "Random_start:75\n",
      "30136\n",
      "0.742413278773\n",
      "episode is 191\n",
      "Random_start:79\n",
      "30150\n",
      "0.742310748091\n",
      "episode is 192\n",
      "Random_start:20\n",
      "30182\n",
      "0.742076446142\n",
      "episode is 193\n",
      "Random_start:22\n",
      "30227\n",
      "0.741747085853\n",
      "episode is 194\n",
      "Random_start:68\n",
      "30249\n",
      "0.741586119201\n",
      "episode is 195\n",
      "Normal_start:1\n",
      "30303\n",
      "0.741191169343\n",
      "episode is 196\n",
      "Random_start:50\n",
      "30326\n",
      "0.741023014712\n",
      "episode is 197\n",
      "Random_start:27\n",
      "30406\n",
      "0.740438430165\n",
      "episode is 198\n",
      "Random_start:40\n",
      "30422\n",
      "0.740321569366\n",
      "episode is 199\n",
      "Random_start:18\n",
      "30460\n",
      "0.740044099892\n",
      "episode is 200\n",
      "Random_start:11\n",
      "30553\n",
      "0.739365474489\n",
      "episode is 201\n",
      "Normal_start:1\n",
      "30726\n",
      "0.738104763048\n",
      "episode is 202\n",
      "Random_start:73\n",
      "30825\n",
      "0.737384296022\n",
      "episode is 203\n",
      "Random_start:10\n",
      "30889\n",
      "0.736918919009\n",
      "episode is 204\n",
      "Random_start:30\n",
      "30926\n",
      "0.736650008761\n",
      "episode is 205\n",
      "Normal_start:1\n",
      "30979\n",
      "0.736264986296\n",
      "episode is 206\n",
      "Random_start:36\n",
      "31095\n",
      "0.735423007354\n",
      "episode is 207\n",
      "Normal_start:1\n",
      "31167\n",
      "0.734900890773\n",
      "episode is 208\n",
      "Random_start:42\n",
      "31246\n",
      "0.734328445216\n",
      "episode is 209\n",
      "Random_start:79\n",
      "31249\n",
      "0.734306715688\n",
      "episode is 210\n",
      "Random_start:66\n",
      "31303\n",
      "0.733915695647\n",
      "episode is 211\n",
      "Random_start:39\n",
      "31367\n",
      "0.733452537828\n",
      "episode is 212\n",
      "Random_start:32\n",
      "31401\n",
      "0.733206605776\n",
      "episode is 213\n",
      "Random_start:22\n",
      "31456\n",
      "0.732808951507\n",
      "episode is 214\n",
      "Normal_start:1\n",
      "31590\n",
      "0.73184103616\n",
      "episode is 215\n",
      "Random_start:16\n",
      "31632\n",
      "0.731537926583\n",
      "episode is 216\n",
      "Random_start:26\n",
      "31681\n",
      "0.731184459605\n",
      "episode is 217\n",
      "Normal_start:1\n",
      "31766\n",
      "0.730571713269\n",
      "episode is 218\n",
      "Normal_start:1\n",
      "31824\n",
      "0.730153902852\n",
      "episode is 219\n",
      "Random_start:33\n",
      "31898\n",
      "0.729621186093\n",
      "episode is 220\n",
      "Random_start:69\n",
      "31924\n",
      "0.729434108906\n",
      "episode is 221\n",
      "Normal_start:1\n",
      "32040\n",
      "0.728600049188\n",
      "episode is 222\n",
      "Random_start:95\n",
      "32066\n",
      "0.728413237461\n",
      "episode is 223\n",
      "Normal_start:1\n",
      "32133\n",
      "0.727932061804\n",
      "episode is 224\n",
      "Normal_start:1\n",
      "32261\n",
      "0.727013696644\n",
      "episode is 225\n",
      "Random_start:98\n",
      "32274\n",
      "0.726920490922\n",
      "episode is 226\n",
      "Normal_start:1\n",
      "32347\n",
      "0.726397329941\n",
      "episode is 227\n",
      "Normal_start:1\n",
      "32405\n",
      "0.725981939964\n",
      "episode is 228\n",
      "Normal_start:1\n",
      "32515\n",
      "0.72519479284\n",
      "episode is 229\n",
      "Random_start:7\n",
      "32534\n",
      "0.725058918738\n",
      "episode is 230\n",
      "Normal_start:1\n",
      "32580\n",
      "0.724730067277\n",
      "episode is 231\n",
      "Random_start:40\n",
      "32680\n",
      "0.724015694456\n",
      "episode is 232\n",
      "Normal_start:1\n",
      "32764\n",
      "0.723416173107\n",
      "episode is 233\n",
      "Normal_start:1\n",
      "32817\n",
      "0.723038162717\n",
      "episode is 234\n",
      "Random_start:84\n",
      "32891\n",
      "0.722510709658\n",
      "episode is 235\n",
      "Random_start:82\n",
      "32911\n",
      "0.722368221765\n",
      "episode is 236\n",
      "Random_start:66\n",
      "32921\n",
      "0.722296988505\n",
      "episode is 237\n",
      "Normal_start:1\n",
      "32993\n",
      "0.721784319256\n",
      "episode is 238\n",
      "Normal_start:1\n",
      "33061\n",
      "0.721300470446\n",
      "episode is 239\n",
      "Random_start:73\n",
      "33158\n",
      "0.720610843513\n",
      "episode is 240\n",
      "Normal_start:1\n",
      "33247\n",
      "0.719978681216\n",
      "episode is 241\n",
      "Random_start:41\n",
      "33291\n",
      "0.719666359312\n",
      "episode is 242\n",
      "Random_start:94\n",
      "33340\n",
      "0.719318707978\n",
      "episode is 243\n",
      "Normal_start:1\n",
      "33386\n",
      "0.718992496407\n",
      "episode is 244\n",
      "Normal_start:1\n",
      "33520\n",
      "0.718043082711\n",
      "episode is 245\n",
      "Normal_start:1\n",
      "33587\n",
      "0.71756885273\n",
      "episode is 246\n",
      "Normal_start:1\n",
      "33691\n",
      "0.716833363644\n",
      "episode is 247\n",
      "Random_start:33\n",
      "33724\n",
      "0.716600147117\n",
      "episode is 248\n",
      "Random_start:77\n",
      "33737\n",
      "0.716508295068\n",
      "episode is 249\n",
      "Random_start:56\n",
      "33791\n",
      "0.716126883579\n",
      "episode is 250\n",
      "Random_start:81\n",
      "33824\n",
      "0.715893900152\n",
      "episode is 251\n",
      "Random_start:57\n",
      "33850\n",
      "0.715710391595\n",
      "episode is 252\n",
      "Normal_start:1\n",
      "33893\n",
      "0.71540700136\n",
      "episode is 253\n",
      "Random_start:21\n",
      "33943\n",
      "0.715054386021\n",
      "episode is 254\n",
      "Random_start:70\n",
      "33972\n",
      "0.714849949893\n",
      "episode is 255\n",
      "Normal_start:1\n",
      "34066\n",
      "0.714187702246\n",
      "episode is 256\n",
      "Normal_start:1\n",
      "34119\n",
      "0.713814581649\n",
      "episode is 257\n",
      "Random_start:34\n",
      "34158\n",
      "0.71354014748\n",
      "episode is 258\n",
      "Random_start:23\n",
      "34230\n",
      "0.713033780888\n",
      "episode is 259\n",
      "Random_start:88\n",
      "34254\n",
      "0.712865073026\n",
      "episode is 260\n",
      "Normal_start:1\n",
      "34307\n",
      "0.712492653238\n",
      "episode is 261\n",
      "Random_start:12\n",
      "34362\n",
      "0.712106388511\n",
      "episode is 262\n",
      "Random_start:29\n",
      "34380\n",
      "0.711980020734\n",
      "episode is 263\n",
      "Random_start:57\n",
      "34430\n",
      "0.711629118457\n",
      "episode is 264\n",
      "Random_start:40\n",
      "34550\n",
      "0.710787668486\n",
      "episode is 265\n",
      "Normal_start:1\n",
      "34643\n",
      "0.710136238916\n",
      "episode is 266\n",
      "Random_start:35\n",
      "34680\n",
      "0.709877236426\n",
      "episode is 267\n",
      "Normal_start:1\n",
      "34713\n",
      "0.709646315042\n",
      "episode is 268\n",
      "Random_start:82\n",
      "34789\n",
      "0.709114785849\n",
      "episode is 269\n",
      "Normal_start:1\n",
      "34829\n",
      "0.708835195857\n",
      "episode is 270\n",
      "Random_start:28\n",
      "34851\n",
      "0.708681469024\n",
      "episode is 271\n",
      "Normal_start:1\n",
      "34938\n",
      "0.708073880485\n",
      "episode is 272\n",
      "Random_start:76\n",
      "34963\n",
      "0.707899383828\n",
      "episode is 273\n",
      "Random_start:61\n",
      "35022\n",
      "0.707487744637\n",
      "episode is 274\n",
      "Random_start:76\n",
      "35060\n",
      "0.707222749647\n",
      "episode is 275\n",
      "Normal_start:1\n",
      "35110\n",
      "0.70687422541\n",
      "episode is 276\n",
      "Random_start:42\n",
      "35162\n",
      "0.706511945014\n",
      "episode is 277\n",
      "Normal_start:1\n",
      "35200\n",
      "0.706247320757\n",
      "episode is 278\n",
      "Random_start:32\n",
      "35287\n",
      "0.705641849006\n",
      "episode is 279\n",
      "Random_start:87\n",
      "35317\n",
      "0.705433187752\n",
      "episode is 280\n",
      "Normal_start:1\n",
      "35364\n",
      "0.705106410952\n",
      "episode is 281\n",
      "Random_start:9\n",
      "35459\n",
      "0.704446373429\n",
      "episode is 282\n",
      "Normal_start:1\n",
      "35522\n",
      "0.704009009998\n",
      "episode is 283\n",
      "Random_start:20\n",
      "35573\n",
      "0.703655155643\n",
      "episode is 284\n",
      "Random_start:31\n",
      "35637\n",
      "0.703211358374\n",
      "episode is 285\n",
      "Random_start:95\n",
      "35650\n",
      "0.703121246755\n",
      "episode is 286\n",
      "Random_start:32\n",
      "35701\n",
      "0.702767845044\n",
      "episode is 287\n",
      "Normal_start:1\n",
      "35748\n",
      "0.702442320661\n",
      "episode is 288\n",
      "Normal_start:1\n",
      "35828\n",
      "0.701888588327\n",
      "episode is 289\n",
      "Normal_start:1\n",
      "35898\n",
      "0.701404435788\n",
      "episode is 290\n",
      "Random_start:69\n",
      "35909\n",
      "0.701328385483\n",
      "episode is 291\n",
      "Normal_start:1\n",
      "35974\n",
      "0.700879168044\n",
      "episode is 292\n",
      "Normal_start:1\n",
      "36022\n",
      "0.70054762562\n",
      "episode is 293\n",
      "Random_start:47\n",
      "36063\n",
      "0.700264559126\n",
      "episode is 294\n",
      "Random_start:20\n",
      "36093\n",
      "0.700057510817\n",
      "episode is 295\n",
      "Random_start:40\n",
      "36112\n",
      "0.699926412345\n",
      "episode is 296\n",
      "Normal_start:1\n",
      "36183\n",
      "0.699436738447\n",
      "episode is 297\n",
      "Random_start:12\n",
      "36273\n",
      "0.698816524521\n",
      "episode is 298\n",
      "Random_start:95\n",
      "36305\n",
      "0.698596138496\n",
      "episode is 299\n",
      "Normal_start:1\n",
      "36349\n",
      "0.698293222842\n",
      "11\n",
      "0.5 0.5\n",
      "0.5 1.5\n",
      "12\n",
      "0.5 1.5\n",
      "1.5 1.5\n",
      "22\n",
      "1.5 1.5\n",
      "1.5 2.5\n",
      "32\n",
      "1.5 2.5\n",
      "1.5 3.5\n",
      "42\n",
      "1.5 3.5\n",
      "1.5 4.5\n",
      "43\n",
      "1.5 4.5\n",
      "2.5 4.5\n",
      "53\n",
      "2.5 4.5\n",
      "2.5 5.5\n",
      "63\n",
      "2.5 5.5\n",
      "2.5 6.5\n",
      "73\n",
      "2.5 6.5\n",
      "2.5 7.5\n",
      "74\n",
      "2.5 7.5\n",
      "3.5 7.5\n",
      "75\n",
      "3.5 7.5\n",
      "4.5 7.5\n",
      "76\n",
      "4.5 7.5\n",
      "5.5 7.5\n",
      "77\n",
      "5.5 7.5\n",
      "6.5 7.5\n",
      "87\n",
      "6.5 7.5\n",
      "6.5 8.5\n",
      "88\n",
      "6.5 8.5\n",
      "7.5 8.5\n",
      "89\n",
      "7.5 8.5\n",
      "8.5 8.5\n",
      "99\n",
      "8.5 8.5\n",
      "8.5 9.5\n",
      "100\n",
      "8.5 9.5\n",
      "9.5 9.5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+UFPWZ7/H30/OLUQyMZqJh0CU3WZRcSTR6RRY1jGjU\njetwYI/GXE10c0N+eI0JLjnAOSZk70lwl8jd5OCN15Ng3OOPZFaEScyNxgio6DjREfxJBCOgMyCI\nzmjQcWCc5/7RPTo9ItBdRdW3ez6vc/pU13e6qj628PCd6up6zN0REZHykkk7gIiIxE/FXUSkDKm4\ni4iUIRV3EZEypOIuIlKGVNxFRMrQfou7mS01sx1m9vSgscPN7F4z25hb1h3cmCIiUogDmbn/Ejh3\nyNhc4D53/1vgvty6iIgEwg7kS0xmNg64y92Pz60/B0x1921m9lFgtbsfezCDiojIgasscrsj3X1b\n7vnLwJEf9EIzmwXMAhgxYsRJxxxzTJGHjEd/fz+ZTPofNYSQI4QMoeQIIUMoOULIEEqOEDIAbNiw\nYae71xe0kbvv9wGMA54etN495OddB7Kf8ePHe9pWrVqVdgR3DyNHCBncw8gRQgb3MHKEkME9jBwh\nZHB3Bx7zA6ixgx/F/pO0PXc6htxyR5H7ERGRg6DY4v4b4Mu5518GWuKJIyIicTiQSyFvB1qBY82s\nw8y+AlwLnG1mG4GzcusiIhKI/X6g6u4Xf8CPpsWcRUREYpL+x8AiIhI7FXcRkTJU7HXuIiLBWtGy\njEVtPWztH82YTDdzJtUyvWlmsiGebIb7/gVe74BRY2Ha9+BTFyZ2eM3cRSQ2R61aR2PrMwA0tj7D\nUavWJZ5hRcsy5rUanf11ONnlvFZjRcuy5EI82Qy//Ra8/hLg2eVvv5UdT4hm7iISi1MeegqA9W/v\nySvqpzz0FH+aMjGxHIvaeuihjlmVnbyT2UUn2VusrGzfQ9drNxW0r+7ubjZt2lR4iI7H4J3PA3AU\nr3Ae98OenuxMPqHZu2buIhKLP02ZSM2QsZrceJK29o8GYDRGLfbu+O4k57Lv9O59/PWOxCJo5i4i\nsdnSeELerH1L4wmJZxiT6aazv46j+z4BwBLeAqAh08UNl19S0L5Wr17N1KlTCw/xv6/LnZIZYtTY\nwvdVJM3cRaSszJlUSy35M+daepkzqTa5ENO+B1VDjldVmx1PiIq7iMSqLpO/TNr0ppksnOxU0wdk\nZ+wLJ3uyV8t86kL4h5/CqKMByy7/4aeJXi2j0zIiEqv1n03+VMxQ05tmsuPlJwF46GuN6YT41IWJ\nFvOhNHMXESlDKu4iImVIxV1EpAxFKu5mdpWZPW1mz5jZt+MKJSIi0RRd3M3seOCrwCnAp4HzzewT\ncQUTEZHiRZm5TwDa3P0td+8D7gdmxBNLRESisGzv1SI2NJtAtr3eZKAHuI9sE9crh7xuFjALoL6+\n/qTm5uRunLM3u3btYuTIkalmCCVHCBlCyRFChlByhJAhjhwNbdm5a+ek/tQyxKWxsbHd3U8uaKNC\nO2oPfgBfAdqBB4CfAf++r9ePHz/+4LUHP0ChdDMPIUcIGdzDyBFCBvcwcoSQwT16ju03POHbb3gi\n1QxxITtxLqg+R/pA1d1/4e4nufsZQBewIcr+REQkHpG+oWpmH3H3HWZ2DNnz7afGE0tERKKIevuB\nZWZ2BLAHuMLdu2PIJCIiEUUq7u5+elxBREQkPvqGqohIGVJxFxEpQ7rlr0iZWNGyjEVtPWztH82Y\nTDdzJtUmew/zQDIM5Kjd9CF2U8m1829JLUeaSmbmvmDBApYsWQLAkiVLWLBgQbqBRHLuW/lxHm49\nD4CHW8/jvpUfTzzDipZlzGs1OvvrcLLLea3GipZlwyrD4BwDPVPTypG2kpi5L168GICdO3fmFfXF\nixcze/bslFKJwINrzgCgp2dDXlF/cM0ZnH7aA4nlWNTWQw91fOvFF6mormProUcA1Tx+bx/+/OMF\n7au7u5+u9sK2AVi76RAu8EpG7roff+cVuiqy335/5M499K57tOD9dXd3s/3+uwvP8cJ2zqWKDx9+\nNnsyVVBVRQ81LGrrYnpTwbsrWSUxc589ezYVFRV5YxUVFSrskrpsAa8eMlqdaGEH2No/GoBRu3cx\nou+9/qG7Pbn52wcda3fCc8iB4+3o62Jb3xvvjm/tH5VojrSVxMwd4JprrsmbtV9zzTXphREZZNqZ\n6/Nm7dPOXJ94hjGZbjr76xi5bTUAvz79m0C2f+i1V59d0L5Wr17N1KmfKTjDj+ffQmd/HRw2OW+8\nIdPF4u9fUvD+sjmmFrzdT3M5lg8ZH5N5veB9lbKSmLmLyL7NmVRLLb15Y7X0MmdS7bDKEFKOtJVU\nca+pqclbioQikxmdt0za9KaZLJzs1FgfkJ0tL5zsiV4hMpChIdOF0Z9KhpBypK1kTssAzJs3L+0I\nInvVOLU97QhMb5rJljtaALjoR4WfBokrQwgfWoaSI00lNXMXEZEDo+IuIlKGVNxFRMpQpOJuZt8x\ns2fM7Gkzu93MRsQVTEREild0cTezBuBbwMnufjxQAXwhrmAiIlK8qKdlKoFaM6sEDgG2Ro8kIiJR\nWbb3apEbm10F/BDoAf7g7v99L6+ZBcwCqK+vP6m5ubno48UhlG7mIeQIIUMoOULIEEeOuuuy92Hq\nurr4W3OUy3tRLhkAGhsb29395II2KrSj9sADqANWAvVAFbACuGRf24wfP/4g9QY/cKF0Mw8hRwgZ\n3MPIEUIG9+g5Nl9yqW++5NJUM8QlhBwhZHB3Bx7zAmt0lNMyZwGb3P0Vd98D3An8XYT9iYhITKIU\n9xeBU83sEDMzYBqQ/B2TRETkfYou7u7eBtwBPA48ldvXjTHlEhGRCCLdW8bdvw98P6YsIiISE31D\nVUSkDKm4i4iUoZK65a9khdBhPoQMoeQIIcNADjZtp9cr+eL8W1LLIWHQzL3EhNBhPoQMoeQIIcPg\nHL25PqZp5eiY+yAdi7PNsDsWP0rH3AcTPb68RzP3EjPQ5f5jk3fwWnU9O96qpxf4dl8fN6/dWNC+\nun0kPylwG4C1fUfR+98qqXhxF5k39pDpeYceavhu6x5uf7m14P11d/fws+cK327tpgp2U8X3Kv+D\nT2a2vDte3d4Hry0taF8ndHfDpsK7KB2zaTs3VVfyqyN6ea66ms1+JAALtvbRcvddBe+vu7ubm+++\nueDt1m7dDsdUkhmxDfqzDbt7qGFRW1diTSs6rn0k+2TH23lFvePaRxg799RkQsi7NHMvMQNd7g+r\nepMRFYO63FORWIYP6mafVpf7948H8F54wu9F7nib62vYdMR7X5ff2j8qsQxj557K+976ClTYU6KZ\ne4kZ6HJ/zlu/BWDRY98Csj0zl88s7C/R6tWrmXpiYberAJjyn7ku90M0ZLr59dfOL3h/2S73kwvP\nkety/y99XxqSo4uHLi+szdy61auZOnVqwRmuzmVgW/54Q6aLm75ZeKu71UXmGHgvrh+XPz4m83rB\n+4pi7A9Pz5u1j/3h6YkeX96jmXuJCaGzewgZQskRQoaQckg4VNxLzOAu94ary33KOULIEFIOAEZk\n8peSCp2WKUHTm2bS/nj2Kohv/qO63KedI4QMIeUYu2BK2hEEzdxFRMqSiruISBmK0kP1WDNbN+jx\nhpl9O85wIiJSnKLPubv7c8AJAGZWAXQCy2PKJSIiEcR1WmYa8Bd337LfV4qIyEEXV3H/AnB7TPsS\nEZGILNt7NcIOzKqBrcB/dffte/n5LGAWQH19/UnNzc2RjhdVKN3Mo+Z4p//fAKjIfDe1DHEJIUcI\nGULJEUKGUHKEkAGgsbGx3d0L+zp5oR21hz6AJuAPB/La8ePHH4S+4IUJpZt51ByPtV/sj7VfnGqG\nuISQI4QM7mHkCCGDexg5Qsjg7g485gXW5jhOy1yMTsmIiAQlUnE3s0OBs4E744kjIiJxiNog+03g\niJiyiIhITPQNVRGRMqTiLiJShlTcS9CKlmWs3bSdthdeZcr8WxLvkyki4VNxLzGDGyE76TVClvdc\n//WV3PaDbA/Y237QyvVfX5lKjusuOp+brr4CgJuuvoLrLiq8K5aUDxX3EpNtkF3Dh+xtPmRvAwON\nkHtSTjY8/XL+GgC6tvVw/ddX0rWtJ288KTde8U8AvNaxhesuOp/XOrbkjcvwo+JeYgYaZO9683B2\nvXn4oPHkGiHLey770WlkhlxzlqnMjidp1vVLyVRWDclRxazrlyaaQ8Kh4l5ixmS6AXji+dN44vnT\nBo0n2whZ3vONJWfucz0p37l1+T7XZXhRcS8xaoQsIgdCxb3EDDRCrmYPpNQgW96v+pBM3jK1HIeO\nzFvK8KUG2SVoetNMul67CYAbLk+nQbbk++riqWlHAODKpb9KO4IEQjN3EZEypOIuIlKGVNxFRMpQ\n1Fv+jjazO8zsz2a23swmxxVMRESKF/UD1Z8Ad7v7P+ba7R0SQyYREYmo6OJuZqOAM4DLANx9N7A7\nnlgiIhJF0Q2yzewE4EbgWeDTQDtwVa6Bx+DXqUH2Qcixdu1aAE488cTUMsQlhBwhZAglRwgZQskR\nQgZIuEE2cDLQB0zKrf8E+F/72kYNst8TNcfSpUt96dKlqWaISwg5QsjgHkaOEDK4h5EjhAzuyTfI\n7gA63L0tt34H8JkI+xMRkZgUXdzd/WXgJTM7Njc0jewpGhERSVnUq2WuBG7NXSnzAnB59EgiIhJV\npOLu7uvInnsXEZGA6BuqIiJlSMVdRKQMqbiXoBUty1i7aTttm15lyvxb1BxbRN5Hxb3ErGhZxrxW\nYzdVgNHZX8e8VlOBT9HEmycyY/kMAGYsn8HEmyemkmP9cRPYeEETABsvaGL9cRNSySFhUHEvMYva\neuihhgaMBgyAHmpY1NaTcrLh6ZzmcwDY+MZGJt48kY1vbMwbT8pz084CoG/DBtYfN4G+DRvyxmX4\nUXEvMVv7RwNwRP9hHNF/2KDxUWlFGtbuufAeqq06b6zaqrnnwnsSzXHsfX+E6vwcVFdnx2VYUnEv\nMWMy3QAc3fcJju77xKDx19OKNOy1f6l9n+tJmfDkE/tcl+FFxb3EzJlUSy29eWO19DJnUm1KiUQk\nRCruJWZ600wWTnaq6QOgIdPFwsnO9KaZKScb3kZVjcpbpmb06PylDFtRbz8gKZjeNJMdLz8JwENf\na0w5jQCs+eKatCMAMOGR1rQjSCA0cxcRKUMq7iIiZUjFXUSkDEU6525mm4G/Au8AfV5oGygRETko\n4vhAtdHdd8awHxERiYlOy4iIlCHL9l4tcmOzTcDrZE/L/F93v3Evr5kFzAKor68/qbm5uejjxSGU\nbuZRczS0Zf9d7pzUn1qGuISQI4QMoeQIIUMoOULIANDY2Nhe8GnvQjtqD34ADbnlR4AngDP29frx\n48cfpN7gBy6UbuZRc2y/4QnffsMTqWaISwg5QsjgHkaOEDK4h5EjhAzu7sBjXmB9jnRaxt07c8sd\nwHLglCj7ExGReBRd3M3sUDM7bOA58Dng6biCiYhI8aJcLXMksNzMBvZzm7vfHUsqERGJpOji7u4v\nAJ+OMYuIiMREl0KKiJQhFXcRkTKkW/4WYEXLMha19bC1fzRjMt3MmVSbyn3UV7Qso3bTh9hNJdfO\nvyW1HCISLs3cD9CKlmXMazU6++twsst5rcaKlmWp5Nid+3c5rRwiEjbN3A/QorYeeqjj4u2t1O55\nha6K7Dd7H7lzD73rHi14f93d3Wy/v/CLi9a+sJ1zqeLDh5/NnkwVVFXRQw2L2rqY3lTw7kSkTKm4\nH6Ct/dm2ZYf0Q+WgOzbsTvgtHDjejr4u3s6MgKojcvlSbu8mIkFRcT9AYzLddPbXsWvkZwFYfthu\nINvDdPH3Lyl4f6tXr2bq1KkFb/fT+bfQ2V/H8vfle73gfYlI+dI59wM0Z1IttfTmjdXSy5xJtcMy\nh4iETcX9AE1vmsnCyU6N9WFkZ+wLJ3viV6kM5GjIdGH0p5ZDRMKm0zIFmN40E3/+cQCuvfrsVHPo\nw1MR2RfN3EVEypCKu4hIGVJxFxEpQ5GLu5lVmNlaM7srjkAiIhJdHDP3q4D1MexHRERiEqm4m9lY\n4PPAz+OJIyIicbBs79UiNza7A1gIHAb8s7ufv5fXzAJmAdTX15/U3Nxc9PHiELWb+ab7+gH42LRo\nv/SE0FU9hAyh5AghQyg5QsgQSo4QMgA0Nja2u/vJBW1UaEftgQdwPvB/cs+nAnftb5vx48cfrObg\nByxqN/M7f9zud/64PfUccQghg3sYOULI4B5GjhAyuIeRI4QM7u7AY15gjY4y/ZwCXGBmm4FfAWea\n2S0R9iciIjEpuri7+zx3H+vu44AvACvdvfA7aImISOx0nbuISBmK5d4y7r4aWB3HvkREJDrN3EVE\nypCKu4hIGVJxL8CKlmWs3bSdthdeZcr8W9SUWkSCVTLF/ahV62hsfQaAxtZnOGrVukSPv6JlGfNa\njV6vxIHO/jrmtVoqBX7c3N/xucWrAPjc4lWMm/u7xDOwYBQsmZx9vmRydl1EglESxf2Uh54CYP3b\nezhq1TrWv70nbzwJi9p66KGGMW++ypg3XwWghxoWtfUklgHgtGv/CMCGHW8xbu7v2LDjrbzxRCz+\nVHa589lsUd/5bP64iKSuJIr7n6ZMpGbIWE1uPClb+0cDMHLXS4zc9dKg8WRnrGvmnkV1Rf5YdUV2\nPDGzn4SKIf9HKmqy4yIShJIo7gBbGk/Y5/rBNibTDcDIbasZuW31oPHXE80BsOGHn9/neiKu2bHv\ndRFJVckU97TNmVRLLb15Y7X0MmdSbUqJREQ+WEkV97pM/jJJ05tmsnCyU2N9ADRkulg42ZneNDP5\nMMDoERV5y1TUHJ6/FJFgxPIN1aSs/2yyp2KGmt40ky13tABw0Y/SvY3OugXnpnp8AOZtSjuBiHyA\nkpq5i4jIgVFxFxEpQ0UXdzMbYWZ/MrMnzOwZM/tBnMFERKR4Uc659wJnuvsuM6sC1pjZ7939kZiy\niYhIkYou7rnWT7tyq1W5R/ENWUVEJDaRzrmbWYWZrQN2APe6e1s8sUREJArLTsAj7sRsNLAcuNLd\nnx7ys1nALID6+vqTmpubIx8viqjdzOuuWwxA19WzU80RhxAyhJIjhAyh5AghQyg5QsgA0NjY2O7u\nJxe0UaEdtT/oAXwP+Od9vWb8+PFxNwUvWNRu5psvudQ3X3Jp6jniEEIG9zByhJDBPYwcIWRwDyNH\nCBnc3YHHvMCaHOVqmfrcjB0zqwXOBv5c7P5ERCQ+Ua6W+Shws5lVkD133+zud8UTS0REoohytcyT\nwIkxZhERkZjoG6oiImVIxV1EpAypuBdgoEH2I2qQLSKBK5nift1F53PT1VcAcNPVV3DdRecnevzB\nDbIh3QbZHXMfpGPxo9nnix+lY+6DiWeQ8CxYsIAlS5YAsGTJEhYsWJBuoBTdt/LjPNx6HgAPt57H\nfSs/nnKi5JVEcb/xin8C4LWOLVx30fm81rElbzwJAw2yM9WvkqlOr0F2x7W5W/fseDtb1He8nT8u\nw9Lixdkv1+3cuZMFCxawc+fOvPHh5ME1ZwDQ07OB+1Z+nJ6eDXnjw0VJFPdZ1y8lU1mVN5aprGLW\n9UsTyzDQIHvzkcbmI23QeLINssfOPRWGNl+qyI3LsDV79mwqKvL/YFRUVDB7drRvUpei0097AKge\nMlqdGx8+SqK4A3zn1uX7XD/YBhpkLz2jnqVn1A8aT75B9tgfnr7PdRmerrnmmn2uDyfTzly/z/Xh\noGSKe9rUIFtESklJFffqQ0fmLZM0uEG2kX6DbEZk8pciQE1NTd5yOMtkRucth5uSapB95dJfpXr8\n6U0zabk7e4eFm76ZboPssQumpHp8CdO8efPSjhCMxqntaUdIlaZ9IiJlSMVdRKQMqbiLiJShKPdz\nP9rMVpnZs2b2jJldFWcwEREpXpQPVPuAq939cTM7DGg3s3vd/dmYsomISJGKnrm7+zZ3fzz3/K/A\neqAhrmAiIlK8uBpkjwMeAI539zeG/KysGmT/5OWfAHDVUdHOQoXQeDeEDKHkCCFDKDlCyBBKjhAy\nQEoNsoGRQDswY3+vLYcG2Zf9/jK/7PeXpZ4jDiFkcA8jRwgZ3MPIEUIG9zByhJDBPeEG2QBmVgUs\nA2519zuj7EtEROIT5WoZA34BrHf34XdfURGRgEWZuU8BLgXONLN1ucffx5RLREQiKPpSSHdfA9h+\nXygiIonTN1RFRMqQiruISBkqmVv+rmhZxqK2Hrb2j2ZMpps5k2oTv5f6ipZlrN26nd1eyZT5t6SS\nQUTkQJTEzH1FyzLmtRqd/XU42eW8VmNFy7LEM/R6JQ6pZJAwXf/1ldz2g1YAbvtBK9d/fWXKiURK\nZOa+qK2HHur4XuV/8MnMlnfHq9v74LXCmmSf0N0NmwrvzHLMpu3cVF3JokwPb/kI1gM91LCorYvp\nTQXvTsrEL+evAaBrW09eUf/l/DVc9qPT0oolUhoz9639ey/Gu6nY6/jBsDv37+DRvRnqe0e8O761\nf1RiGSQ8l/3oNDJDpkiZSlTYJXUlMXMfk+mms7+Of+n7Ut54Q6aLhy4vrN3dutWrmTp1asEZrp5/\nC539dbBtaLbXC96XlJdvLDkzb9b+jSVnpphGJKskZu5zJtVSS2/eWC29zJlUO6wyiIgcqJIo7tOb\nZrJwslPNHsBpyHSxcLIneqXKQIaGTBdGfyoZJFzVh2TyliJpK4nTMpAtrre/nL0i4ddfOz+1DPrw\nVPbmq4unph1BJI+mGSIiZUjFXUSkDKm4i4iUoajNOpaa2Q4zezquQCIiEl3UmfsvgXNjyCEiIjGK\nVNzd/QHgtZiyiIhITCzbezXCDszGAXe5+/Ef8PNZwCyA+vr6k5qbm4s+1sK2HgDmRfjiUCjdzEPI\nEUKGUHKEkCGUHCFkCCVHCBkAGhsb29395II2KrSj9tAHMA54+kBeO378+EgdwC+84WG/8IaHI+0j\nlG7mIeQIIYN7GDlCyOAeRo4QMriHkSOEDO7uwGNeYG3W1TIiImVIxV1EpAxFvRTydqAVONbMOszs\nK/HEEhGRKCLdW8bdL44riIiIxEenZUREypCKu4hIGSqZ4r5ibSdrX+ymbdNrTLl2JSvWdqYdSUQk\nWCVR3Fes7WTenU+x+51+ADq7e5h351PDtsBPvHkiM5bPAGDG8hlMvHliyonSs/64CWy8IHuT/Y0X\nNLH+uAkpJxIJQ0kU90X3PEfPnnfyxnr2vMOie55LKVF6zmk+B4CNb2xk4s0T2fjGxrzx4eS5aWcB\n0LdhA+uPm0Dfhg154yLDWUkU963dPQWNl7N7LryHaqvOG6u2au658J6UEqXn2Pv+CNX57wXV1dlx\nkWGuJIr7mNF7v5fMB42Xu/Yvte9zfTiZ8OQT+1wXGa5KorjPOedYaqsq8sZqqyqYc86xKSUSEQlb\nSRT36Sc2sHDGRBpG12JAw+haFs6YyPQTG9KOlppRVaPylsPa6NH5SxGJ9g3VJE0/sWFYF/Oh1nxx\nTdoRgjHhkda0I4gEpyRm7iIiUhgVdxGRMqTiLiJShqLe8vdcM3vOzJ43s7lxhRIRkWiKLu5mVgFc\nD5wHfBK42Mw+GVcwEREpXpSZ+ynA8+7+grvvBn4FNMUTS0REoohyKWQD8NKg9Q5g0tAXmdksYFZu\ntdfMno5wzDh8GNiZcgYII0cIGSCMHCFkgDByhJABwsgRQgaAgr+xedCvc3f3G4EbAczsMXc/+WAf\nc19CyBBKjhAyhJIjhAyh5AghQyg5QsgwkKPQbaKclukEjh60PjY3JiIiKYtS3B8F/tbMPmZm1cAX\ngN/EE0tERKIo+rSMu/eZ2f8E7gEqgKXu/sx+Nrux2OPFKIQMEEaOEDJAGDlCyABh5AghA4SRI4QM\nUEQOc/eDEURERFKkb6iKiJQhFXcRkTKUSHEP4TYFZrbUzHakeZ29mR1tZqvM7Fkze8bMrkopxwgz\n+5OZPZHL8YM0cuSyVJjZWjO7K8UMm83sKTNbV8wlZzFlGG1md5jZn81svZlNTiHDsbn3YODxhpl9\nO4Uc38n9uXzazG43sxFJZ8jluCqX4Zkk34e91SozO9zM7jWzjbll3X535O4H9UH2w9a/AP8FqAae\nAD55sI+7lxxnAJ8Bnk762IMyfBT4TO75YcCGlN4LA0bmnlcBbcCpKb0ns4HbgLtS/P+yGfhwWsfP\nZbgZ+B+559XA6JTzVAAvA3+T8HEbgE1AbW69Gbgshf/+44GngUPIXnjyR+ATCR37fbUK+Ddgbu75\nXOBf97efJGbuQdymwN0fAF5L+rhDMmxz98dzz/8KrCf7hznpHO7uu3KrVblH4p+sm9lY4PPAz5M+\ndkjMbBTZv9C/AHD33e7enW4qpgF/cfctKRy7Eqg1s0qyxXVrChkmAG3u/pa79wH3AzOSOPAH1Kom\nshMAcsvp+9tPEsV9b7cpGPYtlcxsHHAi2VlzGsevMLN1wA7gXndPI8e/A98F+lM49mAO/NHM2nO3\ny0jax4BXgJtyp6h+bmaHppBjsC8Atyd9UHfvBH4MvAhsA1539z8knYPsrP10MzvCzA4B/p78L20m\n7Uh335Z7/jJw5P420AeqKTCzkcAy4Nvu/kYaGdz9HXc/gew3i08xs+OTPL6ZnQ/scPf2JI/7AU7L\nvRfnAVeY2RkJH7+S7K/hP3P3E4E3yf7qnYrclxIvAP4zhWPXkZ2lfgwYAxxqZpckncPd1wP/CvwB\nuBtYB7yTdI698ey5mf3+pp1EcddtCgYxsyqyhf1Wd78z7Ty5X/9XAecmfOgpwAVmtpnsqbozzeyW\nhDMA784WcfcdwHKypxKT1AF0DPrt6Q6yxT4t5wGPu/v2FI59FrDJ3V9x9z3AncDfpZADd/+Fu5/k\n7mcAXWQ/I0vLdjP7KEBuuWN/GyRR3HWbghwzM7LnVde7++IUc9Sb2ejc81rgbODPSWZw93nuPtbd\nx5H9M7HS3ROfoZnZoWZ22MBz4HNkfyVPjLu/DLxkZgN3/psGPJtkhiEuJoVTMjkvAqea2SG5vy/T\nyH42lTiid7AUAAAA1ElEQVQz+0hueQzZ8+23pZEj5zfAl3PPvwy07G+DJO4KWcxtCmJnZrcDU4EP\nm1kH8H13/0XCMaYAlwJP5c53A8x39/+XcI6PAjfnGq5kgGZ3T+1SxJQdCSzP1hEqgdvc/e4UclwJ\n3JqbAL0AXJ5ChoF/4M4GvpbG8d29zczuAB4H+oC1pHcLgGVmdgSwB7giqQ+591argGuBZjP7CrAF\nuHC/+8ldWiMiImVEH6iKiJQhFXcRkTKk4i4iUoZU3EVEypCKu4hIGVJxFxEpQyruIiJl6P8DSd6T\nRCORvDkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21bc561ff28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "Alpha=0.1 #学习速率\n",
    "Beta=0.9 #对未来奖励的衰减度\n",
    "Epsilon_start=1#选择随机动作的概率 Epsilon\n",
    "Epsilon_stop=0.01\n",
    "decay_rate=0.00001\n",
    "N_Mesh=10\n",
    "N_States=100#假设网格为5*5\n",
    "Actions=['left','right','up','down']\n",
    "Obstacle=[3,5,16,21,23,27,29,31,33,34,56,57,66,67,78,81,86,98]\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Caculate_coordinate(State):\n",
    "    X_Axis=0.5+(State-1)%N_Mesh\n",
    "    Y_Axis=0.5+((State-1)//N_Mesh)\n",
    "    return X_Axis,Y_Axis\n",
    "def Initial_Q_Table (N_States,Actions):#以状态数和动作为变量，返回Q表\n",
    "    Q_Table=np.zeros([N_States,len(Actions)])#初始化Q_表\n",
    "    Q_Table=pd.DataFrame(Q_Table,index=np.arange(1,N_States+1),columns=Actions)#Q_表定义行列标签\n",
    "    print ('初始化Q表完成') \n",
    "    return Q_Table\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Select_Action(State,Q_Table,ep):#根据目前的状态选择动作,返回执行动作\n",
    "    Epsilon=Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*ep)\n",
    "    States_Actions=Q_Table.loc[[State],:]#选择目前状态下，取出Q表的目前的状态行,state选择为数字\n",
    "    if(np.random.rand()<Epsilon or np.all(States_Actions==[0])):#其实np.all(States_Action==[0])也行\n",
    "        Execute_Action=np.random.choice(Actions)\n",
    "    else:\n",
    "        #Q_T=Q_Table.T\n",
    "        #G=(Q_Table.T).loc[:,[State]]\n",
    "        Execute_Action=np.array(pd.DataFrame.idxmax((Q_Table.T).loc[:,[State]]))[0]#这里真心累，有毒\n",
    "    return Execute_Action\n",
    "#———————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Next_State(State,Execute_Action):#根据目前状态个选择的动作,返回即时奖励和新状态\n",
    "    if(Execute_Action=='left'):\n",
    "        #目标在右上角，如果Execute_Action为‘left’，不可能到达目标\n",
    "        if(State%N_Mesh==1):#判断是不是在网格最左边\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+1) in Obstacle):\n",
    "            State=State-1\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State-1\n",
    "            R=1\n",
    "    elif(Execute_Action=='right'):\n",
    "        if(State==N_States-1):\n",
    "            State='End'\n",
    "            R=100\n",
    "        elif(State%N_Mesh==0):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+1) in Obstacle):\n",
    "            State=State+1\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State+1\n",
    "            R=1\n",
    "    elif(Execute_Action=='up'):\n",
    "        if(State==N_States-N_Mesh):\n",
    "            State='End'\n",
    "            R=100\n",
    "        elif(N_States+1-N_Mesh<=State<=N_States-1):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State+10) in Obstacle):\n",
    "            State=State+10\n",
    "            R=-100\n",
    "        else:\n",
    "            State=State+N_Mesh\n",
    "            R=1\n",
    "    else:\n",
    "        if(0<=State<=N_Mesh):\n",
    "            State=State\n",
    "            R=-100\n",
    "        elif((State-10) in Obstacle):\n",
    "            State=State-10\n",
    "            R=-100\n",
    "        else:\n",
    "            R=0\n",
    "            State=State-10\n",
    "    return State,R\n",
    "        \n",
    "#————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Run_Function():\n",
    "    #——————————————————————————————————————————————————画图程序部分\n",
    "    \n",
    "    \n",
    "    #——————————————————————————————————————————————————\n",
    "    Q_Table=Initial_Q_Table (N_States,Actions)\n",
    "    Step=0 \n",
    "    for episode in np.arange(300) :#跑20回合\n",
    "        if(episode%300==0):\n",
    "            print('Q_Table,%d'%episode)\n",
    "            Q_Table\n",
    "#         fig=plt.figure()\n",
    "#         ax=fig.gca()\n",
    "#         ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "#         ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "#         ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "#         plt.grid()\n",
    "        print('episode is %d' % episode)\n",
    "        if(np.random.rand()<0.4):\n",
    "            State_=1 #定义起点 \n",
    "            print(\"Normal_start:%d\"%State_)\n",
    "        else:\n",
    "            State_=np.random.randint(low=5, high=100)\n",
    "            while(State_ in Obstacle):\n",
    "                State_=np.random.randint(low=5, high=100)\n",
    "            print(\"Random_start:%d\"%State_)\n",
    "        Is_End=False\n",
    "        while not Is_End:\n",
    "            A=Select_Action(State_,Q_Table,Step)\n",
    "            Next_S,R=Next_State(State_,A)\n",
    "            #print(Next_S)\n",
    "            Q_Table_S_A=Q_Table.loc[[State_],[A]]\n",
    "            if Next_S!='End':\n",
    "                D_T=np.array((Q_Table.loc[[Next_S],:]).T)\n",
    "                Q_Target=R+Beta*D_T.max() #Next_S状态中的可选动作中的最大Q值动作\n",
    "                Step+=1\n",
    "            else:\n",
    "                Q_Target=R\n",
    "                Step+=1\n",
    "                Is_End=True #目的是结束循\n",
    "#————————————————————————————————————————————————————————————————————————————画图程序\n",
    "#             X_State,Y_State=Caculate_coordinate(State_)\n",
    "#             if(Next_S)=='End':\n",
    "#                 H=N_States\n",
    "#             else:\n",
    "#                 H=Next_S\n",
    "#             X_Next_State,Y_Next_State=Caculate_coordinate(H)                      \n",
    "#             if(Step==0):\n",
    "#                 plt.scatter(X_State,Y_State)  \n",
    "#             else:\n",
    "#                 plt.scatter(X_State,Y_State) \n",
    "#                 plt.scatter(X_Next_State,Y_Next_State)\n",
    "#                 plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "#————————————————————————————————————————————————————————————————————————————\n",
    "            Q_Table.loc[[State_],[A]]+=Alpha*(Q_Target-Q_Table_S_A)\n",
    "            State_=Next_S\n",
    "        print(Step)\n",
    "        print(Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-decay_rate*Step))\n",
    "        #plt.show()\n",
    "        #print(Q_Table)\n",
    "    return Q_Table\n",
    "#————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Q__Table=Run_Function()\n",
    "Q__Table\n",
    "Final_Q_Table=np.array(Q__Table)\n",
    "#—————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "def Plot_Final_Graph(Q_TABLE):\n",
    "    f=open('A.txt','w+')\n",
    "    f.write(str(Q_TABLE))\n",
    "    f.close()\n",
    "    fig=plt.figure()\n",
    "    ax=fig.gca()\n",
    "    ax.set(xlim=[0, N_Mesh], ylim=[0, N_Mesh])\n",
    "    ax.set_xticks(np.arange(0,(N_Mesh+1)))\n",
    "    ax.set_yticks(np.arange(0,(N_Mesh+1)))\n",
    "    plt.grid()\n",
    "    State_Start=1\n",
    "    State=State_Start\n",
    "    Step=1\n",
    "    while(State!=N_States):\n",
    "        Action=np.argmax((Q_TABLE[State-1]))\n",
    "        if(Action==0):\n",
    "             Next_State=State-1\n",
    "        elif(Action==1):\n",
    "             Next_State=State+1\n",
    "        elif(Action==2):\n",
    "             Next_State=State+N_Mesh\n",
    "        else:\n",
    "             Next_State=State-N_Mesh\n",
    "        Step+=1\n",
    "        print(Next_State)\n",
    "        X_State,Y_State=Caculate_coordinate(State)\n",
    "        X_Next_State,Y_Next_State=Caculate_coordinate(Next_State)\n",
    "        if(Step==0):\n",
    "            plt.scatter(X_State,Y_State)  \n",
    "        else:\n",
    "            plt.scatter(X_State,Y_State) \n",
    "            plt.scatter(X_Next_State,Y_Next_State)\n",
    "            plt.plot([X_State,X_Next_State],[Y_State,Y_Next_State])\n",
    "        for J in np.arange(len(Obstacle)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle[J]))[0],(Caculate_coordinate(Obstacle[J]))[1],marker=\"1\")\n",
    "        print(X_State,Y_State)\n",
    "        print(X_Next_State,Y_Next_State)\n",
    "        State=Next_State\n",
    "    plt.show()\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "Plot_Final_Graph(Final_Q_Table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
